# Append all module-level product ranks, select 12 modules as in DVG, and top UPC by revenue (by channel code) 	
# JHL 

### PRELIMINARIES 
list.of.packages <- c("folderfun", "data.table", "bit64", "lubridate", "foreign")
new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
if(length(new.packages)) install.packages(new.packages,repos = "http://cran.us.r-project.org")

require(folderfun) 
require(data.table)
require(bit64)
require(lubridate)
require(foreign)

# 06-14 
# Set paths 
# Repository contains RMS data from 06-14 merged to module level in RData format as "move"
setff("rms_move")
repository <- ffrms_move()
setff("path_home")
path_home <- ffpath_home()

do_path <- sprintf("%s/do",path_home)
dta_path <- sprintf("%s/raw/nielsen",path_home)
save_path <- sprintf("%s/dta/nielsen/pranks",path_home)

### 2. Append into single file ### 
setwd(save_path)
# List files in repository by descending order of size
csv <- system("ls *.csv -S", intern = TRUE)
csv <- lapply(as.list(csv), function(x) paste(save_path,"/",x,sep=""))	 
pranks <- lapply(csv,fread)
pranks <- lapply(pranks,data.table)
# Convert integer to integer64 (or character) if necessary or rbind fails
pranks <- lapply(pranks,function(x){x[,upc:=as.character(upc)]})
pranks <- rbindlist(pranks,fill=T)

saveRDS(pranks,sprintf('%s/pranks.rds',save_path))
write.dta(pranks, file = sprintf('%s/pranks.dta',save_path))

### 3. Pick top product in the 12 modules selected by DVG, across channel codes (DFM) ###
pranks <- readRDS(sprintf('%s/pranks.rds',save_path))

top_pranks <- pranks[(module==1040|module==1290|module==1303|module==1362|module==1463|module==1484|module==1493|module==3603|module==7080|module==7260|module==7734|module==8404)&rank==1,]

top1_pranks_D <- top_pranks[channel_code=="D",]
top1_pranks_F <- top_pranks[channel_code=="F",]
top1_pranks_M <- top_pranks[channel_code=="M",]

saveRDS(top1_pranks_D, sprintf('%s/top1_pranks_D.rds',save_path))
saveRDS(top1_pranks_F, sprintf('%s/top1_pranks_F.rds',save_path))
saveRDS(top1_pranks_M, sprintf('%s/top1_pranks_M.rds',save_path))